package com.hliushi.crawler;

import org.junit.Test;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * @ClassName JdkApiTest
 * @Description 演示使用JDK自带的API实现网络爬虫
 * @Author hliushi
 * @Date 2021/6/30 16:14
 * @Version V1.0
 **/
public class JdkApiTest {

    @Test
    public void testGet() throws Exception {
        // 1.确定要访问/爬取的URL
        URL url = new URL("http://www.itcast.cn/");

        // 2.获取连接对象
        HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();

        // 3.设置连接信息:请求方式/请求参数/请求头...
        // 请求方式默认就是GET, 注意要大写
        urlConnection.setRequestMethod("GET");
        urlConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59");
        urlConnection.setConnectTimeout(30000); // 设置超时时间, 单位毫秒

        // 4.获取数据
        InputStream in = urlConnection.getInputStream();
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        String line;
        StringBuilder stringBuffer = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            stringBuffer.append(line).append("\n");
        }
        System.out.println("html = " + stringBuffer.toString());

        // 5.关闭资源
        reader.close();
        in.close();
    }

    @Test
    public void testPost() throws Exception {
        // 1.确定要访问/爬取的URL
        URL url = new URL("http://www.itcast.cn/");

        // 2.获取连接对象
        HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();

        // 3.设置连接信息:请求方式/请求参数/请求头...
        // 请求方式默认就是GET, 注意要大写
        urlConnection.setDoOutput(true);    // 允许向url输出内容
        urlConnection.setRequestMethod("POST");
        urlConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59");
        urlConnection.setConnectTimeout(30000); // 设置超时时间, 单位毫秒

        OutputStream out = urlConnection.getOutputStream();
        out.write("username= xx".getBytes());

        // 4.获取数据
        InputStream in = urlConnection.getInputStream();
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        String line;
        StringBuilder stringBuffer = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            stringBuffer.append(line).append("\n");
        }
        System.out.println("html = " + stringBuffer.toString());

        // 5.关闭资源
        reader.close();
        in.close();

    }
}
